In [2]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import random
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier, MLPRegressor
import math
from io import StringIO # Python3
import sys
In [2]:
features = pd.read_csv('ex3data1_X.txt', sep=' ', header=None)
labels = pd.read_csv('ex3data1_T.txt', sep=' ', header=None)
data = pd.concat([features, labels], axis=1, ignore_index=True)
# data = data.sample(frac=1).reset_index(drop=True) # shuffle
data.head()
Out[2]:
0
1
2
3
4
5
6
7
8
9
...
400
401
402
403
404
405
406
407
408
409
0
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
1
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
2
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
3
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
4
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
5 rows × 410 columns
In [3]:
train_list = []
test_list = []
for i in range(10):
train_list.append(data.iloc[i*500 : i*500 + 450])
test_list.append(data.iloc[i*500 + 450 : i*500 + 500])
train = pd.concat(train_list)
test = pd.concat(test_list)
train.head()
Out[3]:
0
1
2
3
4
5
6
7
8
9
...
400
401
402
403
404
405
406
407
408
409
0
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
1
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
2
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
3
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
4
0
0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
...
1
0
0
0
0
0
0
0
0
0
5 rows × 410 columns
In [38]:
mlp = MLPClassifier(hidden_layer_sizes=(100),
activation='logistic',
learning_rate_init=0.3,
solver='sgd',
max_iter=500,
alpha=0,
shuffle=False,
verbose=500,
validation_fraction=0.1,
early_stopping=True)
mlp.fit(train.iloc[:, :400], train.iloc[:, 400:])
Iteration 1, loss = 3.99651326
Validation score: 0.000000
Iteration 2, loss = 2.41861040
Validation score: 0.364444
Iteration 3, loss = 1.47182554
Validation score: 0.620000
Iteration 4, loss = 1.00615274
Validation score: 0.737778
Iteration 5, loss = 0.78050620
Validation score: 0.804444
Iteration 6, loss = 0.65194548
Validation score: 0.808889
Iteration 7, loss = 0.57556251
Validation score: 0.813333
Iteration 8, loss = 0.52377520
Validation score: 0.828889
Iteration 9, loss = 0.48555921
Validation score: 0.846667
Iteration 10, loss = 0.44855021
Validation score: 0.846667
Iteration 11, loss = 0.42015840
Validation score: 0.862222
Iteration 12, loss = 0.39497288
Validation score: 0.857778
Iteration 13, loss = 0.37151568
Validation score: 0.864444
Iteration 14, loss = 0.35473148
Validation score: 0.871111
Iteration 15, loss = 0.33491034
Validation score: 0.864444
Iteration 16, loss = 0.32234912
Validation score: 0.871111
Iteration 17, loss = 0.30593094
Validation score: 0.875556
Iteration 18, loss = 0.29421353
Validation score: 0.868889
Iteration 19, loss = 0.28141734
Validation score: 0.877778
Iteration 20, loss = 0.26935987
Validation score: 0.868889
Iteration 21, loss = 0.25764703
Validation score: 0.875556
Iteration 22, loss = 0.24795440
Validation score: 0.882222
Iteration 23, loss = 0.24224988
Validation score: 0.882222
Iteration 24, loss = 0.23020585
Validation score: 0.886667
Iteration 25, loss = 0.22253420
Validation score: 0.888889
Iteration 26, loss = 0.21121527
Validation score: 0.884444
Iteration 27, loss = 0.20187291
Validation score: 0.884444
Iteration 28, loss = 0.19682451
Validation score: 0.882222
Validation score did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Out[38]:
MLPClassifier(activation='logistic', alpha=0, batch_size='auto', beta_1=0.9,
beta_2=0.999, early_stopping=True, epsilon=1e-08,
hidden_layer_sizes=100, learning_rate='constant',
learning_rate_init=0.3, max_iter=500, momentum=0.9,
nesterovs_momentum=True, power_t=0.5, random_state=None,
shuffle=False, solver='sgd', tol=0.0001, validation_fraction=0.1,
verbose=500, warm_start=False)
In [5]:
# precisei salvar a saída acima em dois '.txt' (epoch_loss e validation_score)
epoch_loss = pd.read_csv('epoch_loss.txt', header=None)
epoch_loss.plot()
Out[5]:
<matplotlib.axes._subplots.AxesSubplot at 0x10dc19080>
In [40]:
validation_error = pd.read_csv('validation_score.txt', header=None)
validation_error = validation_error.apply(lambda x: 1 - x)
validation_error.plot()
Out[40]:
<matplotlib.axes._subplots.AxesSubplot at 0x10eb8e4e0>
In [39]:
mlp.score(test.iloc[:,:400], test.iloc[:,400:])
Out[39]:
0.90000000000000002
O erro no conjunto de treinamento continua diminuindo após 200+ épocas, porém o erro no conjunto de validação diminui até a época 25 e depois começa a aumentar. Isso indica que após a época 25 o modelo começa a sofrer overfitting nos dados de treinamento e, portanto, o treinamento deve ser parado na época 25.
In [92]:
data2 = pd.read_csv('ex3data2.txt', header=None, sep=' ')
data2.head()
Out[92]:
0
1
2
3
4
5
6
7
8
9
10
11
12
13
0
0.00632
18.0
2.31
0
0.538
6.575
65.2
4.0900
1
296
15.3
396.90
4.98
24.0
1
0.02731
0.0
7.07
0
0.469
6.421
78.9
4.9671
2
242
17.8
396.90
9.14
21.6
2
0.02729
0.0
7.07
0
0.469
7.185
61.1
4.9671
2
242
17.8
392.83
4.03
34.7
3
0.03237
0.0
2.18
0
0.458
6.998
45.8
6.0622
3
222
18.7
394.63
2.94
33.4
4
0.06905
0.0
2.18
0
0.458
7.147
54.2
6.0622
3
222
18.7
396.90
5.33
36.2
In [34]:
# standardize data
df_x = data2.iloc[:,:13]
df_y = data2.iloc[:,13]
scaler = StandardScaler().fit(df_x)
df_x = pd.DataFrame(scaler.transform(df_x))
data2 = pd.concat([df_x, df_y], axis=1)
data2.head()
Out[34]:
0
1
2
3
4
5
6
7
8
9
10
11
12
13
0
-0.419782
0.284830
-1.287909
-0.272599
-0.144217
0.413672
-0.120013
0.140214
-0.982843
-0.666608
-1.459000
0.441052
-1.075562
24.0
1
-0.417339
-0.487722
-0.593381
-0.272599
-0.740262
0.194274
0.367166
0.557160
-0.867883
-0.987329
-0.303094
0.441052
-0.492439
21.6
2
-0.417342
-0.487722
-0.593381
-0.272599
-0.740262
1.282714
-0.265812
0.557160
-0.867883
-0.987329
-0.303094
0.396427
-1.208727
34.7
3
-0.416750
-0.487722
-1.306878
-0.272599
-0.835284
1.016303
-0.809889
1.077737
-0.752922
-1.106115
0.113032
0.416163
-1.361517
33.4
4
-0.412482
-0.487722
-1.306878
-0.272599
-0.835284
1.228577
-0.511180
1.077737
-0.752922
-1.106115
0.113032
0.441052
-1.026501
36.2
In [58]:
data2 = data2.sample(frac=1).reset_index(drop=True) # shuffle
train = data2.iloc[:406]
test = data2.iloc[406:]
train.head()
Out[58]:
0
1
2
3
4
5
6
7
8
9
10
11
12
13
0
-0.418126
3.160441
-1.516987
-0.272599
-1.249924
0.140137
-1.169050
2.563455
-0.867883
-0.565640
-0.534275
0.441052
-0.964825
24.7
1
-0.207116
-0.487722
1.231945
3.668398
0.434551
2.161728
1.053485
-0.833960
-0.523001
-0.031105
-1.736418
0.361122
-1.504494
50.0
2
-0.403217
-0.487722
-0.375976
-0.272599
-0.299707
0.630220
0.402727
-0.483566
-0.523001
-0.143951
1.130230
0.417588
-0.453191
27.5
3
-0.402961
-0.487722
2.422565
-0.272599
0.469104
-1.183370
0.857902
-0.938447
-0.637962
1.798194
0.760340
0.421206
0.757911
15.2
4
-0.413480
-0.487722
-1.266023
-0.272599
-0.576134
-0.187534
0.008005
-0.244978
-0.752922
-1.278354
-0.303094
0.333711
0.069658
29.6
In [79]:
mlp = MLPRegressor(hidden_layer_sizes=(64),
activation='logistic',
learning_rate_init=0.0001,
solver='sgd',
max_iter=2000,
alpha=0,
shuffle=False,
verbose=True,
validation_fraction=0.1,
early_stopping=True)
mlp.fit(train.iloc[:, :13], train.iloc[:, 13])
Iteration 1, loss = 288.60702731
Validation score: -5.977450
Iteration 2, loss = 284.17242146
Validation score: -5.826353
Iteration 3, loss = 277.52021630
Validation score: -5.632096
Iteration 4, loss = 269.17677001
Validation score: -5.406930
Iteration 5, loss = 259.66760490
Validation score: -5.160593
Iteration 6, loss = 249.41504233
Validation score: -4.901037
Iteration 7, loss = 238.62225036
Validation score: -4.635088
Iteration 8, loss = 227.59255316
Validation score: -4.367818
Iteration 9, loss = 216.61602528
Validation score: -4.102868
Iteration 10, loss = 205.61408129
Validation score: -3.843661
Iteration 11, loss = 195.07262739
Validation score: -3.591597
Iteration 12, loss = 184.79635917
Validation score: -3.348654
Iteration 13, loss = 174.91767358
Validation score: -3.115973
Iteration 14, loss = 165.46012280
Validation score: -2.894289
Iteration 15, loss = 156.43704720
Validation score: -2.684035
Iteration 16, loss = 147.87815263
Validation score: -2.485279
Iteration 17, loss = 139.75450907
Validation score: -2.297959
Iteration 18, loss = 132.16645012
Validation score: -2.121601
Iteration 19, loss = 125.05770715
Validation score: -1.955888
Iteration 20, loss = 118.33548474
Validation score: -1.800616
Iteration 21, loss = 111.88041194
Validation score: -1.655585
Iteration 22, loss = 106.13813318
Validation score: -1.519439
Iteration 23, loss = 100.56164745
Validation score: -1.392419
Iteration 24, loss = 95.36452421
Validation score: -1.273856
Iteration 25, loss = 90.58877600
Validation score: -1.163055
Iteration 26, loss = 86.16278216
Validation score: -1.059553
Iteration 27, loss = 81.85759810
Validation score: -0.963346
Iteration 28, loss = 78.01018463
Validation score: -0.873447
Iteration 29, loss = 74.46993663
Validation score: -0.789493
Iteration 30, loss = 70.96326337
Validation score: -0.711650
Iteration 31, loss = 67.79470750
Validation score: -0.639096
Iteration 32, loss = 64.86693766
Validation score: -0.571416
Iteration 33, loss = 62.20705357
Validation score: -0.508188
Iteration 34, loss = 59.56208460
Validation score: -0.449546
Iteration 35, loss = 57.13340415
Validation score: -0.394930
Iteration 36, loss = 55.00272059
Validation score: -0.343733
Iteration 37, loss = 52.99771389
Validation score: -0.295918
Iteration 38, loss = 51.01319198
Validation score: -0.251568
Iteration 39, loss = 49.24459702
Validation score: -0.210166
Iteration 40, loss = 47.51786099
Validation score: -0.171688
Iteration 41, loss = 46.02072934
Validation score: -0.135593
Iteration 42, loss = 44.55337767
Validation score: -0.101978
Iteration 43, loss = 43.19803005
Validation score: -0.070589
Iteration 44, loss = 41.96995113
Validation score: -0.041210
Iteration 45, loss = 40.79290737
Validation score: -0.013792
Iteration 46, loss = 39.70878627
Validation score: 0.011816
Iteration 47, loss = 38.64837198
Validation score: 0.035659
Iteration 48, loss = 37.70274853
Validation score: 0.058014
Iteration 49, loss = 36.81939670
Validation score: 0.078978
Iteration 50, loss = 35.93959594
Validation score: 0.098535
Iteration 51, loss = 35.19513340
Validation score: 0.117021
Iteration 52, loss = 34.46832935
Validation score: 0.134401
Iteration 53, loss = 33.78880287
Validation score: 0.150727
Iteration 54, loss = 33.16265615
Validation score: 0.166082
Iteration 55, loss = 32.57077777
Validation score: 0.180499
Iteration 56, loss = 31.96050554
Validation score: 0.193921
Iteration 57, loss = 31.42756794
Validation score: 0.206597
Iteration 58, loss = 30.98650339
Validation score: 0.218707
Iteration 59, loss = 30.50001160
Validation score: 0.230038
Iteration 60, loss = 30.04683998
Validation score: 0.240682
Iteration 61, loss = 29.62465498
Validation score: 0.250737
Iteration 62, loss = 29.26211992
Validation score: 0.260330
Iteration 63, loss = 28.88953503
Validation score: 0.269373
Iteration 64, loss = 28.53367895
Validation score: 0.277911
Iteration 65, loss = 28.20081218
Validation score: 0.285999
Iteration 66, loss = 27.89367631
Validation score: 0.293701
Iteration 67, loss = 27.61055953
Validation score: 0.301051
Iteration 68, loss = 27.31398558
Validation score: 0.307986
Iteration 69, loss = 27.05451308
Validation score: 0.314619
Iteration 70, loss = 26.82048524
Validation score: 0.320997
Iteration 71, loss = 26.59043712
Validation score: 0.327072
Iteration 72, loss = 26.33964763
Validation score: 0.332788
Iteration 73, loss = 26.12268494
Validation score: 0.338263
Iteration 74, loss = 25.91527718
Validation score: 0.343507
Iteration 75, loss = 25.72144944
Validation score: 0.348547
Iteration 76, loss = 25.52854638
Validation score: 0.353381
Iteration 77, loss = 25.35924727
Validation score: 0.358049
Iteration 78, loss = 25.18054486
Validation score: 0.362502
Iteration 79, loss = 24.99959614
Validation score: 0.366743
Iteration 80, loss = 24.84941113
Validation score: 0.370883
Iteration 81, loss = 24.69217582
Validation score: 0.374859
Iteration 82, loss = 24.55088899
Validation score: 0.378708
Iteration 83, loss = 24.40496879
Validation score: 0.382404
Iteration 84, loss = 24.26023874
Validation score: 0.385949
Iteration 85, loss = 24.13807165
Validation score: 0.389408
Iteration 86, loss = 23.99390184
Validation score: 0.392704
Iteration 87, loss = 23.87262915
Validation score: 0.395916
Iteration 88, loss = 23.75338624
Validation score: 0.399039
Iteration 89, loss = 23.63715745
Validation score: 0.402066
Iteration 90, loss = 23.51610439
Validation score: 0.404988
Iteration 91, loss = 23.40922223
Validation score: 0.407840
Iteration 92, loss = 23.30465770
Validation score: 0.410625
Iteration 93, loss = 23.19327986
Validation score: 0.413304
Iteration 94, loss = 23.09153090
Validation score: 0.415918
Iteration 95, loss = 22.99027848
Validation score: 0.418458
Iteration 96, loss = 22.89191989
Validation score: 0.420933
Iteration 97, loss = 22.80031028
Validation score: 0.423360
Iteration 98, loss = 22.69956928
Validation score: 0.425705
Iteration 99, loss = 22.60613332
Validation score: 0.427995
Iteration 100, loss = 22.52113451
Validation score: 0.430275
Iteration 101, loss = 22.43130653
Validation score: 0.432500
Iteration 102, loss = 22.34539804
Validation score: 0.434677
Iteration 103, loss = 22.26295200
Validation score: 0.436815
Iteration 104, loss = 22.17606529
Validation score: 0.438888
Iteration 105, loss = 22.09839064
Validation score: 0.440935
Iteration 106, loss = 22.01093446
Validation score: 0.442916
Iteration 107, loss = 21.93276385
Validation score: 0.444861
Iteration 108, loss = 21.85694177
Validation score: 0.446789
Iteration 109, loss = 21.77950761
Validation score: 0.448672
Iteration 110, loss = 21.70440528
Validation score: 0.450530
Iteration 111, loss = 21.62642413
Validation score: 0.452342
Iteration 112, loss = 21.55650482
Validation score: 0.454151
Iteration 113, loss = 21.49518797
Validation score: 0.455955
Iteration 114, loss = 21.41229416
Validation score: 0.457675
Iteration 115, loss = 21.34390108
Validation score: 0.459372
Iteration 116, loss = 21.27483637
Validation score: 0.461043
Iteration 117, loss = 21.20607727
Validation score: 0.462686
Iteration 118, loss = 21.14182258
Validation score: 0.464318
Iteration 119, loss = 21.07440211
Validation score: 0.465916
Iteration 120, loss = 21.00568088
Validation score: 0.467477
Iteration 121, loss = 20.94127884
Validation score: 0.469029
Iteration 122, loss = 20.88256143
Validation score: 0.470574
Iteration 123, loss = 20.81994182
Validation score: 0.472092
Iteration 124, loss = 20.75226637
Validation score: 0.473564
Iteration 125, loss = 20.69466711
Validation score: 0.475037
Iteration 126, loss = 20.63450124
Validation score: 0.476475
Iteration 127, loss = 20.57212946
Validation score: 0.477890
Iteration 128, loss = 20.51531601
Validation score: 0.479299
Iteration 129, loss = 20.45125720
Validation score: 0.480662
Iteration 130, loss = 20.39787187
Validation score: 0.482045
Iteration 131, loss = 20.33978028
Validation score: 0.483397
Iteration 132, loss = 20.28246820
Validation score: 0.484736
Iteration 133, loss = 20.22716596
Validation score: 0.486055
Iteration 134, loss = 20.17005865
Validation score: 0.487362
Iteration 135, loss = 20.11867867
Validation score: 0.488663
Iteration 136, loss = 20.06099956
Validation score: 0.489926
Iteration 137, loss = 20.00701887
Validation score: 0.491165
Iteration 138, loss = 19.95352861
Validation score: 0.492397
Iteration 139, loss = 19.90420607
Validation score: 0.493630
Iteration 140, loss = 19.84947625
Validation score: 0.494822
Iteration 141, loss = 19.79548050
Validation score: 0.496003
Iteration 142, loss = 19.74967413
Validation score: 0.497193
Iteration 143, loss = 19.69713492
Validation score: 0.498349
Iteration 144, loss = 19.64524959
Validation score: 0.499491
Iteration 145, loss = 19.59388548
Validation score: 0.500618
Iteration 146, loss = 19.54744868
Validation score: 0.501747
Iteration 147, loss = 19.50014700
Validation score: 0.502861
Iteration 148, loss = 19.44813582
Validation score: 0.503947
Iteration 149, loss = 19.40121937
Validation score: 0.505027
Iteration 150, loss = 19.35779189
Validation score: 0.506108
Iteration 151, loss = 19.30654073
Validation score: 0.507155
Iteration 152, loss = 19.26001222
Validation score: 0.508189
Iteration 153, loss = 19.21473692
Validation score: 0.509219
Iteration 154, loss = 19.16721652
Validation score: 0.510231
Iteration 155, loss = 19.12255012
Validation score: 0.511240
Iteration 156, loss = 19.07929206
Validation score: 0.512248
Iteration 157, loss = 19.03368879
Validation score: 0.513247
Iteration 158, loss = 18.99097812
Validation score: 0.514249
Iteration 159, loss = 18.94366025
Validation score: 0.515221
Iteration 160, loss = 18.90155862
Validation score: 0.516199
Iteration 161, loss = 18.86143805
Validation score: 0.517188
Iteration 162, loss = 18.81564139
Validation score: 0.518146
Iteration 163, loss = 18.77210088
Validation score: 0.519087
Iteration 164, loss = 18.72957103
Validation score: 0.520021
Iteration 165, loss = 18.68861367
Validation score: 0.520948
Iteration 166, loss = 18.64675725
Validation score: 0.521866
Iteration 167, loss = 18.60542025
Validation score: 0.522776
Iteration 168, loss = 18.56495532
Validation score: 0.523678
Iteration 169, loss = 18.52649262
Validation score: 0.524583
Iteration 170, loss = 18.48296465
Validation score: 0.525449
Iteration 171, loss = 18.44549702
Validation score: 0.526338
Iteration 172, loss = 18.40306627
Validation score: 0.527207
Iteration 173, loss = 18.36570656
Validation score: 0.528081
Iteration 174, loss = 18.32498851
Validation score: 0.528935
Iteration 175, loss = 18.28780820
Validation score: 0.529793
Iteration 176, loss = 18.24928467
Validation score: 0.530637
Iteration 177, loss = 18.21089143
Validation score: 0.531472
Iteration 178, loss = 18.17142257
Validation score: 0.532295
Iteration 179, loss = 18.13448294
Validation score: 0.533125
Iteration 180, loss = 18.09674790
Validation score: 0.533941
Iteration 181, loss = 18.05956724
Validation score: 0.534758
Iteration 182, loss = 18.02164845
Validation score: 0.535557
Iteration 183, loss = 17.98607620
Validation score: 0.536358
Iteration 184, loss = 17.94850778
Validation score: 0.537145
Iteration 185, loss = 17.91532970
Validation score: 0.537938
Iteration 186, loss = 17.87951321
Validation score: 0.538719
Iteration 187, loss = 17.84228428
Validation score: 0.539477
Iteration 188, loss = 17.80834881
Validation score: 0.540235
Iteration 189, loss = 17.77314544
Validation score: 0.540986
Iteration 190, loss = 17.73782942
Validation score: 0.541715
Iteration 191, loss = 17.70272183
Validation score: 0.542442
Iteration 192, loss = 17.66803349
Validation score: 0.543173
Iteration 193, loss = 17.63405398
Validation score: 0.543896
Iteration 194, loss = 17.60106565
Validation score: 0.544610
Iteration 195, loss = 17.56692984
Validation score: 0.545311
Iteration 196, loss = 17.53538466
Validation score: 0.546029
Iteration 197, loss = 17.50132389
Validation score: 0.546754
Iteration 198, loss = 17.47006526
Validation score: 0.547474
Iteration 199, loss = 17.43473770
Validation score: 0.548175
Iteration 200, loss = 17.40206949
Validation score: 0.548864
Iteration 201, loss = 17.37117422
Validation score: 0.549563
Iteration 202, loss = 17.34085379
Validation score: 0.550261
Iteration 203, loss = 17.30631944
Validation score: 0.550938
Iteration 204, loss = 17.27468711
Validation score: 0.551610
Iteration 205, loss = 17.24325732
Validation score: 0.552278
Iteration 206, loss = 17.21408522
Validation score: 0.552949
Iteration 207, loss = 17.18257723
Validation score: 0.553612
Iteration 208, loss = 17.15092634
Validation score: 0.554259
Iteration 209, loss = 17.11942527
Validation score: 0.554899
Iteration 210, loss = 17.09086885
Validation score: 0.555544
Iteration 211, loss = 17.06279856
Validation score: 0.556189
Iteration 212, loss = 17.03055848
Validation score: 0.556810
Iteration 213, loss = 17.00290914
Validation score: 0.557437
Iteration 214, loss = 16.97153000
Validation score: 0.558048
Iteration 215, loss = 16.94119220
Validation score: 0.558641
Iteration 216, loss = 16.91238992
Validation score: 0.559260
Iteration 217, loss = 16.88480251
Validation score: 0.559877
Iteration 218, loss = 16.85704611
Validation score: 0.560499
Iteration 219, loss = 16.82665280
Validation score: 0.561091
Iteration 220, loss = 16.79755985
Validation score: 0.561695
Iteration 221, loss = 16.76926610
Validation score: 0.562291
Iteration 222, loss = 16.74224149
Validation score: 0.562896
Iteration 223, loss = 16.71308550
Validation score: 0.563487
Iteration 224, loss = 16.68566886
Validation score: 0.564077
Iteration 225, loss = 16.65941466
Validation score: 0.564669
Iteration 226, loss = 16.63132113
Validation score: 0.565249
Iteration 227, loss = 16.60364116
Validation score: 0.565820
Iteration 228, loss = 16.57838116
Validation score: 0.566384
Iteration 229, loss = 16.54910412
Validation score: 0.566936
Iteration 230, loss = 16.52301827
Validation score: 0.567484
Iteration 231, loss = 16.49566526
Validation score: 0.568030
Iteration 232, loss = 16.47087371
Validation score: 0.568575
Iteration 233, loss = 16.44470738
Validation score: 0.569112
Iteration 234, loss = 16.41853278
Validation score: 0.569645
Iteration 235, loss = 16.39325912
Validation score: 0.570181
Iteration 236, loss = 16.36697808
Validation score: 0.570708
Iteration 237, loss = 16.34124214
Validation score: 0.571237
Iteration 238, loss = 16.31656054
Validation score: 0.571767
Iteration 239, loss = 16.29160298
Validation score: 0.572286
Iteration 240, loss = 16.26640399
Validation score: 0.572797
Iteration 241, loss = 16.24088612
Validation score: 0.573299
Iteration 242, loss = 16.21718773
Validation score: 0.573805
Iteration 243, loss = 16.19159583
Validation score: 0.574301
Iteration 244, loss = 16.16780553
Validation score: 0.574796
Iteration 245, loss = 16.14343669
Validation score: 0.575285
Iteration 246, loss = 16.12033954
Validation score: 0.575781
Iteration 247, loss = 16.09533139
Validation score: 0.576266
Iteration 248, loss = 16.07095123
Validation score: 0.576749
Iteration 249, loss = 16.05069742
Validation score: 0.577246
Iteration 250, loss = 16.02360839
Validation score: 0.577715
Iteration 251, loss = 16.00147385
Validation score: 0.578194
Iteration 252, loss = 15.97673572
Validation score: 0.578665
Iteration 253, loss = 15.95458636
Validation score: 0.579139
Iteration 254, loss = 15.93312058
Validation score: 0.579610
Iteration 255, loss = 15.90918830
Validation score: 0.580073
Iteration 256, loss = 15.88534239
Validation score: 0.580531
Iteration 257, loss = 15.86402987
Validation score: 0.580995
Iteration 258, loss = 15.84157565
Validation score: 0.581452
Iteration 259, loss = 15.81838418
Validation score: 0.581905
Iteration 260, loss = 15.79591032
Validation score: 0.582352
Iteration 261, loss = 15.77477651
Validation score: 0.582797
Iteration 262, loss = 15.75314668
Validation score: 0.583241
Iteration 263, loss = 15.73156201
Validation score: 0.583686
Iteration 264, loss = 15.71073570
Validation score: 0.584127
Iteration 265, loss = 15.68765201
Validation score: 0.584555
Iteration 266, loss = 15.66696900
Validation score: 0.584985
Iteration 267, loss = 15.64510213
Validation score: 0.585410
Iteration 268, loss = 15.62485245
Validation score: 0.585833
Iteration 269, loss = 15.60450569
Validation score: 0.586250
Iteration 270, loss = 15.58497542
Validation score: 0.586666
Iteration 271, loss = 15.56270412
Validation score: 0.587065
Iteration 272, loss = 15.54096192
Validation score: 0.587457
Iteration 273, loss = 15.52462434
Validation score: 0.587862
Iteration 274, loss = 15.50192035
Validation score: 0.588250
Iteration 275, loss = 15.48090564
Validation score: 0.588626
Iteration 276, loss = 15.46225072
Validation score: 0.589015
Iteration 277, loss = 15.44190669
Validation score: 0.589400
Iteration 278, loss = 15.42098606
Validation score: 0.589780
Iteration 279, loss = 15.40119078
Validation score: 0.590169
Iteration 280, loss = 15.38246056
Validation score: 0.590560
Iteration 281, loss = 15.36269855
Validation score: 0.590958
Iteration 282, loss = 15.34437074
Validation score: 0.591357
Iteration 283, loss = 15.32405552
Validation score: 0.591739
Iteration 284, loss = 15.30578290
Validation score: 0.592121
Iteration 285, loss = 15.28641797
Validation score: 0.592517
Iteration 286, loss = 15.26640773
Validation score: 0.592898
Iteration 287, loss = 15.24850179
Validation score: 0.593280
Iteration 288, loss = 15.22918040
Validation score: 0.593658
Iteration 289, loss = 15.21020874
Validation score: 0.594030
Iteration 290, loss = 15.19124686
Validation score: 0.594395
Iteration 291, loss = 15.17381477
Validation score: 0.594763
Iteration 292, loss = 15.15475944
Validation score: 0.595121
Iteration 293, loss = 15.13783604
Validation score: 0.595486
Iteration 294, loss = 15.11837667
Validation score: 0.595836
Iteration 295, loss = 15.10194592
Validation score: 0.596189
Iteration 296, loss = 15.08303925
Validation score: 0.596534
Iteration 297, loss = 15.06535242
Validation score: 0.596881
Iteration 298, loss = 15.04856796
Validation score: 0.597226
Iteration 299, loss = 15.03010930
Validation score: 0.597559
Iteration 300, loss = 15.01334188
Validation score: 0.597902
Iteration 301, loss = 14.99550001
Validation score: 0.598239
Iteration 302, loss = 14.98107333
Validation score: 0.598582
Iteration 303, loss = 14.96168847
Validation score: 0.598908
Iteration 304, loss = 14.94444932
Validation score: 0.599227
Iteration 305, loss = 14.92682897
Validation score: 0.599542
Iteration 306, loss = 14.91104741
Validation score: 0.599862
Iteration 307, loss = 14.89466978
Validation score: 0.600184
Iteration 308, loss = 14.87708376
Validation score: 0.600497
Iteration 309, loss = 14.86096617
Validation score: 0.600813
Iteration 310, loss = 14.84521808
Validation score: 0.601127
Iteration 311, loss = 14.82732341
Validation score: 0.601434
Iteration 312, loss = 14.81267376
Validation score: 0.601745
Iteration 313, loss = 14.79635742
Validation score: 0.602056
Iteration 314, loss = 14.77953677
Validation score: 0.602352
Iteration 315, loss = 14.76324056
Validation score: 0.602650
Iteration 316, loss = 14.74721797
Validation score: 0.602960
Iteration 317, loss = 14.73151019
Validation score: 0.603265
Iteration 318, loss = 14.71576567
Validation score: 0.603573
Iteration 319, loss = 14.69993172
Validation score: 0.603880
Iteration 320, loss = 14.68434745
Validation score: 0.604183
Iteration 321, loss = 14.66983306
Validation score: 0.604495
Iteration 322, loss = 14.65279274
Validation score: 0.604797
Iteration 323, loss = 14.63744206
Validation score: 0.605095
Iteration 324, loss = 14.62215604
Validation score: 0.605390
Iteration 325, loss = 14.60799563
Validation score: 0.605697
Iteration 326, loss = 14.59237040
Validation score: 0.605997
Iteration 327, loss = 14.57692706
Validation score: 0.606290
Iteration 328, loss = 14.56139043
Validation score: 0.606580
Iteration 329, loss = 14.54615210
Validation score: 0.606869
Iteration 330, loss = 14.53244332
Validation score: 0.607161
Iteration 331, loss = 14.51719742
Validation score: 0.607450
Iteration 332, loss = 14.50460281
Validation score: 0.607738
Iteration 333, loss = 14.48823310
Validation score: 0.608012
Iteration 334, loss = 14.47336909
Validation score: 0.608280
Iteration 335, loss = 14.45872655
Validation score: 0.608550
Iteration 336, loss = 14.44682817
Validation score: 0.608831
Iteration 337, loss = 14.43140651
Validation score: 0.609098
Iteration 338, loss = 14.41673302
Validation score: 0.609361
Iteration 339, loss = 14.40355540
Validation score: 0.609626
Iteration 340, loss = 14.38899160
Validation score: 0.609880
Iteration 341, loss = 14.37494632
Validation score: 0.610138
Iteration 342, loss = 14.36169993
Validation score: 0.610394
Iteration 343, loss = 14.34812936
Validation score: 0.610651
Iteration 344, loss = 14.33422604
Validation score: 0.610908
Iteration 345, loss = 14.32109350
Validation score: 0.611157
Iteration 346, loss = 14.30676483
Validation score: 0.611415
Iteration 347, loss = 14.29373294
Validation score: 0.611676
Iteration 348, loss = 14.28136317
Validation score: 0.611939
Iteration 349, loss = 14.26695637
Validation score: 0.612192
Iteration 350, loss = 14.25397327
Validation score: 0.612443
Iteration 351, loss = 14.24150305
Validation score: 0.612692
Iteration 352, loss = 14.22786798
Validation score: 0.612937
Iteration 353, loss = 14.21440982
Validation score: 0.613180
Iteration 354, loss = 14.20249172
Validation score: 0.613427
Iteration 355, loss = 14.18933075
Validation score: 0.613670
Iteration 356, loss = 14.17592170
Validation score: 0.613907
Iteration 357, loss = 14.16372474
Validation score: 0.614144
Iteration 358, loss = 14.15085361
Validation score: 0.614372
Iteration 359, loss = 14.13930177
Validation score: 0.614606
Iteration 360, loss = 14.12555689
Validation score: 0.614831
Iteration 361, loss = 14.11312974
Validation score: 0.615060
Iteration 362, loss = 14.10137586
Validation score: 0.615290
Iteration 363, loss = 14.08948432
Validation score: 0.615516
Iteration 364, loss = 14.07652592
Validation score: 0.615731
Iteration 365, loss = 14.06437997
Validation score: 0.615949
Iteration 366, loss = 14.05222057
Validation score: 0.616168
Iteration 367, loss = 14.04058270
Validation score: 0.616379
Iteration 368, loss = 14.02902637
Validation score: 0.616598
Iteration 369, loss = 14.01691775
Validation score: 0.616810
Iteration 370, loss = 14.00558405
Validation score: 0.617018
Iteration 371, loss = 13.99410193
Validation score: 0.617227
Iteration 372, loss = 13.98128982
Validation score: 0.617428
Iteration 373, loss = 13.96969796
Validation score: 0.617632
Iteration 374, loss = 13.95854961
Validation score: 0.617835
Iteration 375, loss = 13.94694859
Validation score: 0.618039
Iteration 376, loss = 13.93559193
Validation score: 0.618242
Iteration 377, loss = 13.92404152
Validation score: 0.618436
Iteration 378, loss = 13.91253945
Validation score: 0.618642
Iteration 379, loss = 13.90166688
Validation score: 0.618850
Iteration 380, loss = 13.89016370
Validation score: 0.619056
Iteration 381, loss = 13.87879702
Validation score: 0.619256
Iteration 382, loss = 13.86784903
Validation score: 0.619468
Iteration 383, loss = 13.85745500
Validation score: 0.619677
Iteration 384, loss = 13.84656978
Validation score: 0.619879
Iteration 385, loss = 13.83418665
Validation score: 0.620066
Iteration 386, loss = 13.82394033
Validation score: 0.620259
Iteration 387, loss = 13.81279952
Validation score: 0.620452
Iteration 388, loss = 13.80179505
Validation score: 0.620639
Iteration 389, loss = 13.79147738
Validation score: 0.620826
Iteration 390, loss = 13.78069925
Validation score: 0.621019
Iteration 391, loss = 13.77107924
Validation score: 0.621216
Iteration 392, loss = 13.76060621
Validation score: 0.621408
Iteration 393, loss = 13.74855047
Validation score: 0.621586
Iteration 394, loss = 13.73952648
Validation score: 0.621773
Iteration 395, loss = 13.72874388
Validation score: 0.621948
Iteration 396, loss = 13.71804867
Validation score: 0.622126
Iteration 397, loss = 13.70795249
Validation score: 0.622302
Iteration 398, loss = 13.69721212
Validation score: 0.622471
Iteration 399, loss = 13.68705141
Validation score: 0.622649
Iteration 400, loss = 13.67863645
Validation score: 0.622830
Iteration 401, loss = 13.66780536
Validation score: 0.623004
Iteration 402, loss = 13.65689156
Validation score: 0.623171
Iteration 403, loss = 13.64726608
Validation score: 0.623339
Iteration 404, loss = 13.63707914
Validation score: 0.623510
Iteration 405, loss = 13.62780391
Validation score: 0.623682
Iteration 406, loss = 13.61838262
Validation score: 0.623850
Iteration 407, loss = 13.60787372
Validation score: 0.624018
Iteration 408, loss = 13.59832779
Validation score: 0.624188
Iteration 409, loss = 13.58874038
Validation score: 0.624354
Iteration 410, loss = 13.57921562
Validation score: 0.624528
Iteration 411, loss = 13.57040266
Validation score: 0.624702
Iteration 412, loss = 13.56029092
Validation score: 0.624867
Iteration 413, loss = 13.55036932
Validation score: 0.625031
Iteration 414, loss = 13.54093833
Validation score: 0.625192
Iteration 415, loss = 13.53203392
Validation score: 0.625355
Iteration 416, loss = 13.52266486
Validation score: 0.625522
Iteration 417, loss = 13.51330905
Validation score: 0.625676
Iteration 418, loss = 13.50398095
Validation score: 0.625838
Iteration 419, loss = 13.49525629
Validation score: 0.626007
Iteration 420, loss = 13.48534366
Validation score: 0.626166
Iteration 421, loss = 13.47701515
Validation score: 0.626328
Iteration 422, loss = 13.46795495
Validation score: 0.626487
Iteration 423, loss = 13.45938135
Validation score: 0.626646
Iteration 424, loss = 13.45050723
Validation score: 0.626800
Iteration 425, loss = 13.44026920
Validation score: 0.626943
Iteration 426, loss = 13.43201832
Validation score: 0.627086
Iteration 427, loss = 13.42342256
Validation score: 0.627233
Iteration 428, loss = 13.41436718
Validation score: 0.627375
Iteration 429, loss = 13.40627508
Validation score: 0.627520
Iteration 430, loss = 13.39711594
Validation score: 0.627657
Iteration 431, loss = 13.38890371
Validation score: 0.627788
Iteration 432, loss = 13.38068434
Validation score: 0.627937
Iteration 433, loss = 13.37232726
Validation score: 0.628077
Iteration 434, loss = 13.36300657
Validation score: 0.628211
Iteration 435, loss = 13.35507829
Validation score: 0.628350
Iteration 436, loss = 13.34633747
Validation score: 0.628488
Iteration 437, loss = 13.33808482
Validation score: 0.628620
Iteration 438, loss = 13.32964068
Validation score: 0.628755
Iteration 439, loss = 13.32139324
Validation score: 0.628894
Iteration 440, loss = 13.31334711
Validation score: 0.629033
Iteration 441, loss = 13.30485633
Validation score: 0.629165
Iteration 442, loss = 13.29648518
Validation score: 0.629294
Iteration 443, loss = 13.28920354
Validation score: 0.629431
Iteration 444, loss = 13.28102086
Validation score: 0.629562
Iteration 445, loss = 13.27267235
Validation score: 0.629699
Iteration 446, loss = 13.26458642
Validation score: 0.629832
Iteration 447, loss = 13.25659476
Validation score: 0.629965
Iteration 448, loss = 13.24887988
Validation score: 0.630105
Iteration 449, loss = 13.24123029
Validation score: 0.630239
Iteration 450, loss = 13.23352017
Validation score: 0.630375
Iteration 451, loss = 13.22532399
Validation score: 0.630502
Iteration 452, loss = 13.21733871
Validation score: 0.630625
Iteration 453, loss = 13.21055124
Validation score: 0.630757
Iteration 454, loss = 13.20174820
Validation score: 0.630879
Iteration 455, loss = 13.19445486
Validation score: 0.631006
Iteration 456, loss = 13.18664813
Validation score: 0.631132
Iteration 457, loss = 13.17909939
Validation score: 0.631255
Iteration 458, loss = 13.17184259
Validation score: 0.631374
Iteration 459, loss = 13.16453905
Validation score: 0.631495
Iteration 460, loss = 13.15709807
Validation score: 0.631615
Iteration 461, loss = 13.14939079
Validation score: 0.631721
Iteration 462, loss = 13.14274119
Validation score: 0.631846
Iteration 463, loss = 13.13455035
Validation score: 0.631957
Iteration 464, loss = 13.12686079
Validation score: 0.632077
Iteration 465, loss = 13.11998719
Validation score: 0.632201
Iteration 466, loss = 13.11263170
Validation score: 0.632320
Iteration 467, loss = 13.10500395
Validation score: 0.632431
Iteration 468, loss = 13.09791420
Validation score: 0.632549
Iteration 469, loss = 13.09053819
Validation score: 0.632665
Iteration 470, loss = 13.08422254
Validation score: 0.632782
Iteration 471, loss = 13.07646761
Validation score: 0.632894
Iteration 472, loss = 13.06933721
Validation score: 0.633001
Iteration 473, loss = 13.06230500
Validation score: 0.633112
Iteration 474, loss = 13.05515685
Validation score: 0.633224
Iteration 475, loss = 13.04850308
Validation score: 0.633332
Iteration 476, loss = 13.04129417
Validation score: 0.633450
Iteration 477, loss = 13.03418670
Validation score: 0.633565
Iteration 478, loss = 13.02730589
Validation score: 0.633681
Iteration 479, loss = 13.02028474
Validation score: 0.633805
Iteration 480, loss = 13.01377269
Validation score: 0.633929
Iteration 481, loss = 13.00678603
Validation score: 0.634046
Iteration 482, loss = 13.00110510
Validation score: 0.634169
Iteration 483, loss = 12.99279437
Validation score: 0.634273
Iteration 484, loss = 12.98643834
Validation score: 0.634388
Iteration 485, loss = 12.98002495
Validation score: 0.634497
Iteration 486, loss = 12.97379231
Validation score: 0.634605
Iteration 487, loss = 12.96596945
Validation score: 0.634703
Iteration 488, loss = 12.95983189
Validation score: 0.634804
Iteration 489, loss = 12.95401138
Validation score: 0.634908
Iteration 490, loss = 12.94681666
Validation score: 0.635005
Iteration 491, loss = 12.94049239
Validation score: 0.635102
Iteration 492, loss = 12.93418641
Validation score: 0.635198
Validation score did not improve more than tol=0.000100 for two consecutive epochs. Stopping.
Out[79]:
MLPRegressor(activation='logistic', alpha=0, batch_size='auto', beta_1=0.9,
beta_2=0.999, early_stopping=True, epsilon=1e-08,
hidden_layer_sizes=64, learning_rate='constant',
learning_rate_init=0.0001, max_iter=2000, momentum=0.9,
nesterovs_momentum=True, power_t=0.5, random_state=None,
shuffle=False, solver='sgd', tol=0.0001, validation_fraction=0.1,
verbose=True, warm_start=False)
In [80]:
epoch_loss = pd.read_csv('epoch_loss2.txt', header=None)
epoch_loss.plot()
Out[80]:
<matplotlib.axes._subplots.AxesSubplot at 0x113dcd748>
In [81]:
validation_error = pd.read_csv('validation_score2.txt', header=None)
validation_error = validation_error.apply(lambda x: 1 - x)
validation_error.plot()
Out[81]:
<matplotlib.axes._subplots.AxesSubplot at 0x113c35908>
In [91]:
predictions = mlp.predict(test.iloc[:,:13])
pd.DataFrame(predictions, columns=['pred']).plot()
test.iloc[:,13].reset_index(drop=True).plot()
Out[91]:
<matplotlib.axes._subplots.AxesSubplot at 0x114a93c88>
In [ ]:
Content source: abevieiramota/data-science-cookbook
Similar notebooks: